{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import os\n",
    "from scipy.io import loadmat\n",
    "import arff\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "from sklearn.svm import NuSVC, SVC\n",
    "from sklearn.svm import libsvm\n",
    "from imblearn.ensemble import BalancedBaggingClassifier\n",
    "from imblearn.ensemble import RUSBoostClassifier\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "FBANK_feat = os.path.join('FBANK','binnedFeat.mat')\n",
    "FBANK_label = os.path.join('FBANK','binnedLabels.csv')\n",
    "ComParE_feat = os.path.join('baseline','openSMILEall_PCG.mat')\n",
    "BOAW_feat = os.path.join('baseline_boaw','feat','boawFeat.4096.arff')\n",
    "BOAW_meta = os.path.join('baseline_boaw','feat','BOAW_filenames.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def severe2abnormal(labels):\n",
    "    for idx,each in enumerate(labels):\n",
    "        if each == 2:\n",
    "            labels[idx]=1\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.read_csv('/media/taufiq/Data1/heart_sound/heartnet/featGen/mfcc.csv',header=None, delimiter=';')\n",
    "# df.head()\n",
    "# with open('/media/taufiq/Data1/heart_sound/heartnet/featGen/BOAW_filenames.txt','w') as fp:\n",
    "#     for each in df[0].drop_duplicates().values:\n",
    "#         fp.write(\"%s\\n\" % each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3835, 4098)"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load_BOAW:\n",
    "\n",
    "df = arff.load(open(BOAW_feat,'r'))\n",
    "boawData = pd.DataFrame(np.array(df['data'],dtype=float))\n",
    "del df\n",
    "\n",
    "boawFilenames = []\n",
    "with open(BOAW_meta, 'r') as fp:\n",
    "    line = fp.readline()\n",
    "    boawFilenames.append(line.split('\\n')[0])\n",
    "    while line:\n",
    "            line = fp.readline()\n",
    "            boawFilenames.append(line.split('\\n')[0])\n",
    "boawFilenames = boawFilenames[:-1]\n",
    "boawData['filenames'] = boawFilenames\n",
    "boawData.rename({len(boawData.columns)-2:'label'},axis='columns',inplace=True)\n",
    "boawData.label = [int(each) for each in boawData.label]\n",
    "boawData['dataset'] = [each[-1][0] for each in boawData.filenames.str.split('_')]\n",
    "boawData.set_index('filenames',drop=True,inplace=True)\n",
    "\n",
    "del boawFilenames\n",
    "boawData.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3835, 2709)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load FBANK: \n",
    "\n",
    "df = loadmat(FBANK_feat)\n",
    "fbankData = pd.DataFrame(df['feats'])\n",
    "fbankMeta = pd.read_csv(FBANK_label)\n",
    "fbankMeta.file_name = [each[0] for each in fbankMeta.file_name.str.split('.')]\n",
    "fbankData = fbankData.join(fbankMeta)\n",
    "fbankData.dropna(inplace=True)\n",
    "fbankData.rename({'file_name':'filenames'},axis=\"columns\",inplace=True)\n",
    "fbankData.label = [int(each) for each in fbankData.label]\n",
    "fbankData['dataset'] = [each[-1][0] for each in fbankData.filenames.str.split('_')]\n",
    "fbankData.set_index('filenames',drop=True,inplace=True)\n",
    "\n",
    "del df, fbankMeta\n",
    "\n",
    "fbankData.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3835, 6375)"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load ComParE:\n",
    "\n",
    "df = loadmat(ComParE_feat)\n",
    "\n",
    "compareData = pd.DataFrame(df['dataTrain'])\n",
    "compareData = pd.concat((compareData,pd.DataFrame(df['dataDev'])),axis=\"rows\").reset_index(drop=True)\n",
    "compareData['label'] = np.concatenate((np.hstack(df['TrainLabels']),np.hstack(df['DevLabels'])),axis=0)\n",
    "compareData['filenames'] = np.concatenate((np.hstack(df['TrainFiles']),np.hstack(df['DevFiles'])),axis=0)\n",
    "compareData.filenames = [each[0] for each in compareData.filenames.str.split('.')]\n",
    "compareData.label = [int(each) for each in compareData.label]\n",
    "compareData['dataset'] = [each[-1][0] for each in compareData.filenames.str.split('_')]\n",
    "compareData.set_index('filenames',drop=True,inplace=True)\n",
    "\n",
    "del df\n",
    "\n",
    "compareData.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Under-Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM]0.4714285714285714\n",
      "0.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEDCAYAAAA/eB+kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEJpJREFUeJzt3GlsVOXfh/HvVEDzpwuULiogRtTIHxUSiQsY1kKNWNoB\nEVxYCoGgIgFBXlQJcYlEDFETI9KESIKIIQjyQIlESgRNgceYEJDlBYRFtjLDSDst0gq9nxfkaUSW\n2c506I/r84qezpn53cxwdXqYc3zOOScAgDlpqR4AAJAcBB4AjCLwAGAUgQcAowg8ABhF4AHAqDaR\nbnD69GnNnTtXZ8+eVVpamkaPHq3x48erpqZGs2bN0okTJ9SlSxd9+umnysjIaImZAQBR8EX6HHwg\nEFAwGFSPHj1UX1+vkSNH6osvvtCaNWvUoUMHTZkyReXl5aqtrdWcOXNaam4AQAQRD9Hk5uaqR48e\nkqT27dure/fuqq6uVmVlpfx+vyTJ7/dr8+bNyZ0UABCTmI7BHz9+XAcOHFCvXr109uxZ5eTkSLr8\nQyAUCiVlQABAfKIOfH19vWbMmKGysjK1b99ePp/viu//+2sAQGpF/E9WSbp48aJmzJih4uJiFRQU\nSJI6deqkYDConJwcBQIBZWdnR7wf5xw/CGBeTU2N/uenffrPf9rHtN/58/UaMfC/ysrKStJkuNVE\nFfiysjLdf//9mjBhQvO2wYMHa82aNZo6darWrl2rIUOGRLwfn8+nQCAc/7Q3udzcDNbXSnm5ttra\nsJpcGzWpXUz7NbkGBYNhNTZ6/+lly8+ddGusLx4RX0m//fab1q9frx07dqikpER+v1/btm3TlClT\nVFVVpcLCQu3YsUNTp06NawAAQHJEfAf/2GOPaf/+/df83rJly7yeBwDgEc5kBQCjCDwAGEXgAcAo\nAg8ARhF4ADCKwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMIrAA4BRBB4AjCLwAGAU\ngQcAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYBSBBwCjCDwAGEXgAcAoAg8ARhF4ADCK\nwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMIrAA4BRBB4AjCLwAGAUgQcAowg8ABhF\n4AHAKAIPAEYReAAwisADgFEEHgCMihj4srIy9e3bV0VFRc3bPv/8c/Xv319+v19+v1/btm1L6pAA\ngNi1iXSDkSNHaty4cZo7d+4V20tLS1VaWpq0wQAAiYn4Dr5Pnz7KzMy8artzLikDAQC8Efcx+BUr\nVqi4uFhvv/22wuGwlzMBADzgc1G8FT9x4oSmTZum9evXS5JCoZA6duwon8+nTz75RIFAQB9++GHS\nhwVag5qaGm3+36Nqn371b743Ul9Xq4LHuykrKytJk+FWE/EY/LVkZ2c3//mFF17QtGnTot43ELD7\nbj83N4P1tVJerq22Nqy6+gY16UJM+52vb1AwGFZjo/cfbrP83Em3xvriEdUr6d9v8gOBQPOff/zx\nRz344INxPTgAIHkivoOfPXu2du7cqXPnzmngwIF64403tHPnTu3fv19paWnq3Lmz3nvvvZaYFQAQ\ng4iBX7Ro0VXbRo0alZRhAADe4UxWADCKwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgA\nMIrAA4BRBB4AjCLwAGAUgQcAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYBSBBwCjCDwA\nGEXgAcAoAg8ARhF4ADCKwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMIrAA4BRBB4A\njCLwAGAUgQcAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYBSBBwCjCDwAGEXgAcCoiIEv\nKytT3759VVRU1LytpqZGkyZNUmFhoSZPnqxwOJzUIQEAsYsY+JEjR2rp0qVXbCsvL9dTTz2lTZs2\n6YknntCSJUuSNiAAID4RA9+nTx9lZmZesa2yslJ+v1+S5Pf7tXnz5uRMBwCIW1zH4EOhkHJyciRJ\nubm5CoVCng4FAEhcGy/uxOfzRX3b3NwMLx7ypsX6Wi+v1tauXZPS24fUPv2OmPZLU6NycjKUlZWc\nv2PLz51kf33xiCvwnTp1UjAYVE5OjgKBgLKzs6PeNxCw+x+yubkZrK+V8nJttbVh1dU3qEkXYtrv\nfH2DgsGwGhu9/3Cb5edOujXWF4+oXknOuSu+Hjx4sNasWSNJWrt2rYYMGRLXgwMAkidi4GfPnq2x\nY8fq8OHDGjhwoL777jtNnTpVVVVVKiws1I4dOzR16tSWmBUAEIOIh2gWLVp0ze3Lli3zehYAgIc4\nkxUAjCLwAGAUgQcAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYBSBBwCjCDwAGEXgAcAo\nAg8ARhF4ADCKwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMIrAA4BRBB4AjCLwAGAU\ngQcAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYBSBBwCjCDwAGEXgAcAoAg8ARhF4ADCK\nwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMKpNIjsPHjxY6enpSktLU5s2bbR69Wqv\n5gIAJCihwPt8Pi1fvlxZWVlezQMA8EhCh2icc2pqavJqFgCAhxIKvM/n0+TJkzVq1CitWrXKq5kA\nAB5I6BDNypUrlZeXp1AopNLSUt13333q06fPDffJzc1I5CFveqyv9fJqbe3aNSm9fUjt0++Iab80\nNSonJ0NZWcn5O7b83En21xePhAKfl5cnScrOztbQoUO1Z8+eiIEPBMKJPORNLTc3g/W1Ul6urbY2\nrLr6BjXpQkz7na9vUDAYVmOj9x9us/zcSbfG+uIR9yvpr7/+Un19vSTp/Pnz+uWXX/TAAw/Ee3cA\nAI/F/Q4+GAxq+vTp8vl8unTpkoqKivT00097ORsAIAFxB75r165at26dl7MAADzEmawAYBSBBwCj\nCDwAGEXgAcAoAg8ARhF4ADCKwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMIrAA4BR\nBB4AjCLwAGAUgQcAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYBSBBwCjCDwAGEXgAcAo\nAg8ARhF4ADCKwAOAUQQeAIwi8ABgFIEHAKMIPAAYReABwCgCDwBGEXgAMIrAA4BRBB4AjCLwAGAU\ngQcAowg8ABhF4AHAKAIPAEYReAAwKqHAb9u2Tc8884wKCwtVXl7u1UwAAA/EHfimpia9//77Wrp0\nqTZs2KCKigodOnTIy9kAAAmIO/C7d+9Wt27d1LlzZ7Vt21bDhw9XZWWll7MBABIQd+Crq6t11113\nNX+dn5+vM2fOeDIUACBxbVI9QGvR0NCgM2eqb3ib8+fTdfZs3RXbbr/9Dt1xx+3JHK3FtGvXpNra\ncKrHSAov1xYO1+qv8/Ux7/fX+XqFw7WezPBvlp87qfWvLzMzKyn3G3fg8/PzdfLkyeavq6urlZeX\nF3G/3NyMeB8yxTLUpUtOxFt169YCo6RQVlZyXog3Ay/X1rv3fz27L69Yfu4k++uLR9yHaB555BEd\nO3ZMJ06cUGNjoyoqKjRkyBAvZwMAJCDud/C33Xab5s2bp0mTJsk5p+eff17du3f3cjYAQAJ8zjmX\n6iEAAN7jTFYAMIrAA4BRBB4AjEpq4GtqajRp0iQVFhZq8uTJCoev/pzqgQMHNHbsWBUVFam4uFgb\nN25M5kieiHQNnsbGRs2aNUvDhg3TmDFjrvg46c0u0tqWLVum4cOHq7i4WKWlpTp16lQKpoxftNdP\n2rRpkx566CHt3bu3BadLXDTr27hxo4YPH66ioiLNmTOnhSdMTKT1nTp1SuPHj5ff71dxcbG2bt2a\nginjU1ZWpr59+6qoqOi6t/nggw80bNgwFRcXa//+/ZHv1CXRwoULXXl5uXPOuSVLlriPP/74qtsc\nOXLEHT161DnnXHV1tevXr58Lh8PJHCshly5dcgUFBe748eOusbHRjRgxwh08ePCK26xYscLNnz/f\nOedcRUWFmzlzZgomjV00a9u5c6e7cOGCc865b775ptWszbno1uecc3V1de7ll192Y8aMcb///nsK\nJo1PNOs7cuSI8/v9zf/Gzp49m4pR4xLN+ubNm+dWrlzpnHPu4MGDbtCgQakYNS6//vqr27dvn3vu\nueeu+f2ffvrJTZkyxTnn3K5du9zo0aMj3mdS38FXVlbK7/dLkvx+vzZv3nzVbbp166Z77rlHkpSX\nl6dOnTopFAolc6yERHMNnn+uu7CwUNu3b0/FqDGLZm2PP/64br/98pm5vXv3VnX1jc/uvZlEe/2k\nzz77TFOmTFHbtm1TMGX8olnfqlWr9NJLLyk9PV2SlJ2dnYpR4xLN+nw+n+rqLp9NXltbq/z8/FSM\nGpc+ffooMzPzut+vrKxUSUmJJKlXr14Kh8MKBoM3vM+kBj4UCikn5/LZn7m5uRHDvXv3bl28eLE5\n+DejaK7Bc+bMGd15552SLp8vkJmZqXPnzrXonPGI9fpCq1evVv/+/VtiNE9Es759+/bp9OnTGjBg\nQEuPl7Bo1nfkyBEdPnxYL774osaOHauff/65pceMWzTrmz59utatW6cBAwZo2rRpmjdvXkuPmTT/\n7Ip0ef2R3mAlfC2a0tLSa/4UmTlz5lXbfD7fde/nzJkzmjt3rhYuXJjoSDcdZ/BUg3Xr1mnv3r1a\nvnx5qkfxjHNOCxYs0EcffXTFNksuXbqkY8eOacWKFTp58qReeeUVbdiwofkdfWtXUVGhUaNGaeLE\nidq1a5feeustVVRUpHqslEk48F999dV1v9epUycFg0Hl5OQoEAhc99fBuro6TZs2TbNnz9ajjz6a\n6EhJFc01ePLz83X69Gnl5+fr0qVLqqurU4cOHVp61JhFe32hqqoqlZeX6+uvv25VhzEira++vl4H\nDx7UuHHj5JxTMBjUa6+9psWLF6tnz56pGDkm0b42e/furbS0NHXp0kX33nuvjhw5oocffrilx41Z\nNOtbvXq1li5dKunyIcSGhgaFQqFWdSjqevLy8nT69Onmr/+/MTeS1EM0gwcP1po1ayRJa9euvea1\nav7++2+9/vrrKikp0dChQ5M5jieiuQbPoEGDtHbtWknSDz/8oCeffDIVo8YsmrXt27dP8+fP1+LF\ni9WxY8cUTRqfSOtLT0/X9u3bVVlZqS1btqhXr1768ssvW0Xcpeiev4KCAu3cuVPS5UOoR48eVdeu\nXVMxbsyiWd/dd9+tqqoqSdKhQ4fU2NjYquJ+o98YhwwZou+//16StGvXLmVmZjYfAr/RHSbNn3/+\n6SZMmOCGDRvmSktLXU1NjXPOuT179rh33nnHOefcunXrXM+ePV1JSYkrLi52JSUlbv/+/ckcK2Fb\nt251w4YNc0OHDnVLlixxzjn32WefuS1btjjnnGtoaHAzZsxwQ4cOdaNHj3Z//PFHKseNSaS1TZw4\n0fXr16/5+Xr11VdTOW7MIq3vn8aNG9eqPkXjXHTrW7BggXv22WddUVGR27hxY6pGjUuk9R08eNCN\nHTvWjRgxwpWUlLiqqqpUjhuTN9980/Xr18/17NnTDRgwwK1evdqtXLnSffvtt823effdd11BQYEr\nKiqK6rXJtWgAwCjOZAUAowg8ABhF4AHAKAIPAEYReAAwisADgFEEHgCMIvAAYNT/Ab1rSUsnUbDa\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9b26f6c790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = '0'\n",
    "\n",
    "np.random.seed(1)\n",
    "rand_states = np.random.randint(20000,size=(10,))\n",
    "scaler = StandardScaler()\n",
    "mask = compareData.dataset == dataset\n",
    "X_ = scaler.fit_transform(compareData[range(6373)][mask]) ## .4489999\n",
    "results = []\n",
    "for rand_state in rand_states:\n",
    "    rand_state = 144\n",
    "    rus = RandomUnderSampler(random_state=rand_state)    \n",
    "    X,y = rus.fit_resample(X_,compareData.label[mask])\n",
    "    \n",
    "    clf = NuSVC(kernel='linear',probability=True,verbose=True,\n",
    "              random_state=rand_state, nu = .3)\n",
    "    \n",
    "    X_train, X_val, y_train, y_val = train_test_split(X, y,\n",
    "                                                      test_size = 0.2,\n",
    "                                                      random_state = rand_state,\n",
    "                                                      shuffle = True)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    results.append(accuracy_score(clf.predict(X_val), y_val))\n",
    "print(np.mean(results))\n",
    "print(np.std(results))\n",
    "\n",
    "sns.distplot(results,20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(235, 0.7872340425531915),\n",
       " (12172, 0.723404255319149),\n",
       " (5192, 0.7446808510638298),\n",
       " (17289, 0.7446808510638298),\n",
       " (10955, 0.6808510638297872),\n",
       " (7813, 0.6808510638297872),\n",
       " (19279, 0.6808510638297872),\n",
       " (144, 0.851063829787234),\n",
       " (16332, 0.7021276595744681),\n",
       " (7751, 0.7659574468085106)]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zip(rand_states,results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4, error_score='raise-deprecating',\n",
       "       estimator=SVC(C=2, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='linear', max_iter=-1, probability=True, random_state=19433,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'C': [0.001, 0.01, 0.1, 1, 10]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C_grid = [0.0001 0.001, 0.01, 0.1, 1, 10]\n",
    "# gamma_grid = [0.001, 0.01, 0.1, 1, 10]\n",
    "param_grid = {'C': C_grid}\n",
    "\n",
    "grid = GridSearchCV(clf, param_grid, cv = 4, scoring = \"accuracy\")\n",
    "grid.fit(X_train, y_train)\n",
    "\n",
    "print(grid.best_score_)\n",
    "\n",
    "print(grid.best_params_)\n",
    "\n",
    "print(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'compareData' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-248-925c779f3531>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mscaler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mStandardScaler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompareData\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompareData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m6373\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m## .4489999\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompareData\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'compareData' is not defined"
     ]
    }
   ],
   "source": [
    "dataset = '0'\n",
    "\n",
    "\n",
    "scaler = StandardScaler()\n",
    "mask = compareData.dataset == dataset\n",
    "X = scaler.fit_transform(compareData[range(6373)][mask]) ## .4489999\n",
    "y = compareData.label[mask]\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=144,\n",
    "            nu = .5)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y,\n",
    "                                                  test_size = 0.3,\n",
    "                                                  random_state = 114,\n",
    "                                                  shuffle = True)\n",
    "bbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=100,\n",
    "                                oob_score = True,\n",
    "                                random_state=114,\n",
    "                               )\n",
    "bbc.fit(X_train,y_train)\n",
    "print(\"Out of Bag Score &f\" % bbc.oob_score_)\n",
    "print(\"Validation split accuracy &f\" % accuracy_score(bbc.predict(X_val), y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fuse Dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Size 13178\n"
     ]
    }
   ],
   "source": [
    "## Fuse dataframes\n",
    "\n",
    "print(\"Target Size %d\" % (len(boawData.columns)-2+len(fbankData.columns)-2+len(compareData.columns)))\n",
    "fused = boawData[range(len(boawData.columns)-2)].join(fbankData[range(len(fbankData.columns)-2)],rsuffix='fbank')\n",
    "fused = fused.join(compareData,lsuffix='fused')\n",
    "fused.dropna()\n",
    "fused.columns = range(len(fused.columns))\n",
    "fused.rename({len(fused.columns)-2:'label',len(fused.columns)-1:'dataset'},axis=\"columns\",inplace=True)\n",
    "\n",
    "del boawData, fbankData, compareData\n",
    "\n",
    "fused.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Validation Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_state = 114\n",
    "n_estimators = 10\n",
    "dataset = '0'\n",
    "\n",
    "scaler = StandardScaler()\n",
    "mask = fused.dataset == dataset\n",
    "\n",
    "X = scaler.fit_transform(fused[range(len(fused.columns)-2)][mask]) ## .4489999\n",
    "y = fused.label[mask]\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y,\n",
    "                                                  test_size = 0.3,\n",
    "                                                  random_state = rand_state,\n",
    "                                                  shuffle = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dev set of Compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_state = 1\n",
    "n_estimators = 100\n",
    "dataset = '0'\n",
    "\n",
    "scaler = StandardScaler()\n",
    "mask1 = fused.dataset == dataset\n",
    "mask2 = fused[mask1].index.str.contains('train')\n",
    "mask3 = fused[mask1].index.str.contains('devel')\n",
    "\n",
    "X = scaler.fit_transform(fused[range(len(fused.columns)-2)][mask1]) ## .4489999\n",
    "y = fused.label[mask1]\n",
    "\n",
    "X_train = X[mask2]\n",
    "y_train = y[mask2]\n",
    "X_val = X[mask3]\n",
    "y_val = y[mask3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Undersampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out of Bag Score Boaw 0.523904\n",
      "Validation split accuracy Boaw 0.472222\n"
     ]
    }
   ],
   "source": [
    "## BOAW Train\n",
    "del clf\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=rand_state,\n",
    "            nu = .5)\n",
    "\n",
    "boawBbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=n_estimators,\n",
    "                                oob_score = True,\n",
    "                                random_state=rand_state,\n",
    "                                sampling_strategy = 'not minority'\n",
    "                               )\n",
    "boawBbc.fit(X_train[:,:4096],y_train)\n",
    "print(\"Out of Bag Score Boaw %f\" % boawBbc.oob_score_)\n",
    "print(\"Validation split accuracy Boaw %f\" % accuracy_score(boawBbc.predict(X_val[:,:4096]), y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out of Bag Score Fbank 0.376494\n",
      "Validation split accuracy Fbank 0.361111\n"
     ]
    }
   ],
   "source": [
    "## Fbank Train\n",
    "del clf\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=rand_state,\n",
    "            nu = .5)\n",
    "\n",
    "fbankBbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=n_estimators,\n",
    "                                oob_score = True,\n",
    "                                random_state=rand_state,\n",
    "                                sampling_strategy = 'not minority'\n",
    "                               )\n",
    "fbankBbc.fit(X_train[:,4096:4096+2707],y_train)\n",
    "print(\"Out of Bag Score Fbank %f\" % fbankBbc.oob_score_)\n",
    "print(\"Validation split accuracy Fbank %f\" % accuracy_score(fbankBbc.predict(X_val[:,4096:4096+2707]), y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out of Bag Score Compare 0.490040\n",
      "Validation split accuracy Compare 0.522222\n"
     ]
    }
   ],
   "source": [
    "## Compare Train\n",
    "del clf\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=rand_state,\n",
    "            nu = .5)\n",
    "\n",
    "compareBbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=n_estimators,\n",
    "                                oob_score = True,\n",
    "                                random_state=rand_state,\n",
    "                                sampling_strategy = 'not minority',\n",
    "                               )\n",
    "compareBbc.fit(X_train[:,4096+2707:],y_train)\n",
    "print(\"Out of Bag Score Compare %f\" % compareBbc.oob_score_)\n",
    "print(\"Validation split accuracy Compare %f\" % accuracy_score(compareBbc.predict(X_val[:,4096+2707:]), y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy Combination 0.940239\n"
     ]
    }
   ],
   "source": [
    "## Weighted fusion posteriors\n",
    "\n",
    "weights = [.92,.055,.025]\n",
    "\n",
    "softScores = boawBbc.predict_proba(X_train[:,:4096])*weights[1]+fbankBbc.predict_proba(X_train[:,4096:4096+2707])*weights[2]+compareBbc.predict_proba(X_train[:,4096+2707:])*weights[0]\n",
    "print(\"Training accuracy Combination %f\" % accuracy_score(np.argmax(softScores,axis=-1), y_train))\n",
    "\n",
    "\n",
    "softScores = boawBbc.predict_proba(X_val[:,:4096])*weights[1]+fbankBbc.predict_proba(X_val[:,4096:4096+2707])*weights[2]+compareBbc.predict_proba(X_val[:,4096+2707:])*weights[0]\n",
    "print(\"Validation accuracy Combination %f\" % accuracy_score(np.argmax(softScores,axis=-1), y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OverSampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## BOAW Train\n",
    "del clf\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=rand_state,\n",
    "            nu = .5)\n",
    "\n",
    "boawBbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=n_estimators,\n",
    "                                oob_score = True,\n",
    "                                random_state=rand_state,\n",
    "                                sampling_strategy = 'not majority'\n",
    "                               )\n",
    "boawBbc.fit(X_train[:,:4096],y_train)\n",
    "print(\"Out of Bag Score Boaw %f\" % boawBbc.oob_score_)\n",
    "print(\"Validation split accuracy Boaw %f\" % accuracy_score(boawBbc.predict(X_val[:,:4096]), y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Fbank Train\n",
    "del clf\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=rand_state,\n",
    "            nu = .5)\n",
    "\n",
    "fbankBbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=n_estimators,\n",
    "                                oob_score = True,\n",
    "                                random_state=rand_state,\n",
    "                                sampling_strategy = 'not majority'\n",
    "                               )\n",
    "fbankBbc.fit(X_train[:,4096:4096+2707],y_train)\n",
    "print(\"Out of Bag Score Fbank %f\" % fbankBbc.oob_score_)\n",
    "print(\"Validation split accuracy Fbank %f\" % accuracy_score(fbankBbc.predict(X_val[:,4096:4096+2707]), y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Compare Train\n",
    "del clf\n",
    "\n",
    "clf = NuSVC(kernel='linear',probability=True,verbose=False,\n",
    "          random_state=rand_state,\n",
    "            nu = .5)\n",
    "\n",
    "compareBbc = BalancedBaggingClassifier(base_estimator=clf,\n",
    "                                n_estimators=n_estimators,\n",
    "                                oob_score = True,\n",
    "                                random_state=rand_state,\n",
    "                                sampling_strategy = 'not majority',\n",
    "                               )\n",
    "compareBbc.fit(X_train[:,4096+2707:],y_train)\n",
    "print(\"Out of Bag Score Compare %f\" % compareBbc.oob_score_)\n",
    "print(\"Validation split accuracy Compare %f\" % accuracy_score(compareBbc.predict(X_val[:,4096+2707:]), y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Weighted fusion posteriors\n",
    "\n",
    "weights = [.53,.17,.3]\n",
    "\n",
    "softScores = boawBbc.predict_proba(X_train[:,:4096])*weights[1]+fbankBbc.predict_proba(X_train[:,4096:4096+2707])*weights[2]+compareBbc.predict_proba(X_train[:,4096+2707:])*weights[0]\n",
    "print(\"Training accuracy Combination %f\" % accuracy_score(np.argmax(softScores,axis=-1), y_train))\n",
    "\n",
    "\n",
    "softScores = boawBbc.predict_proba(X_val[:,:4096])*weights[1]+fbankBbc.predict_proba(X_val[:,4096:4096+2707])*weights[2]+compareBbc.predict_proba(X_val[:,4096+2707:])*weights[0]\n",
    "print(\"Validation accuracy Combination %f\" % accuracy_score(np.argmax(softScores,axis=-1), y_val))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
